{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基于SVD的协同过滤"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#load数据（用户和物品索引，以及倒排表）\n",
    "import _pickle as cPickle\n",
    "import json  \n",
    "\n",
    "from numpy.random import random\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#用户和item的索引\n",
    "users_index = cPickle.load(open(\"users_index.pkl\", 'rb'))\n",
    "items_index = cPickle.load(open(\"songs_index.pkl\", 'rb'))\n",
    "\n",
    "n_users = len(users_index)\n",
    "n_items = len(items_index)\n",
    "    \n",
    "#用户-物品关系矩阵R\n",
    "#scores = sio.mmread(\"scores\").todense()\n",
    "    \n",
    "#倒排表\n",
    "##每个用户打过分的电影\n",
    "user_items = cPickle.load(open(\"user_songs.pkl\", 'rb'))\n",
    "##对每个电影打过分的事用户\n",
    "item_users = cPickle.load(open(\"song_users.pkl\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5a828f20d611efe2d818ca48edf3b792f4e69e66</td>\n",
       "      <td>SONQBUB12A6D4F8ED0</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a828f20d611efe2d818ca48edf3b792f4e69e66</td>\n",
       "      <td>SONQCXC12A6D4F6A37</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5a828f20d611efe2d818ca48edf3b792f4e69e66</td>\n",
       "      <td>SOOJJCT12A6310E1C0</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5a828f20d611efe2d818ca48edf3b792f4e69e66</td>\n",
       "      <td>SOPKPFW12A6D4F84BC</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5a828f20d611efe2d818ca48edf3b792f4e69e66</td>\n",
       "      <td>SOPVQLJ12A67AE2281</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  rate\n",
       "0  5a828f20d611efe2d818ca48edf3b792f4e69e66  SONQBUB12A6D4F8ED0   0.1\n",
       "1  5a828f20d611efe2d818ca48edf3b792f4e69e66  SONQCXC12A6D4F6A37   0.1\n",
       "2  5a828f20d611efe2d818ca48edf3b792f4e69e66  SOOJJCT12A6310E1C0   0.3\n",
       "3  5a828f20d611efe2d818ca48edf3b792f4e69e66  SOPKPFW12A6D4F84BC   0.9\n",
       "4  5a828f20d611efe2d818ca48edf3b792f4e69e66  SOPVQLJ12A67AE2281   0.4"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取训练数据\n",
    "\n",
    "dpath='./Data/'\n",
    "df_triplet= pd.read_csv(dpath +'RS_Song_train.csv')\n",
    "df_triplet = df_triplet.drop(['play_count'], axis=1)\n",
    "\n",
    "df_triplet['rate']=df_triplet['rate']/10\n",
    "df_triplet.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 初始化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#隐含变量的维数\n",
    "K = 40\n",
    "\n",
    "#item和用户的偏置项\n",
    "bi = np.zeros((n_items,1))    \n",
    "bu = np.zeros((n_users,1))   \n",
    "\n",
    "#item和用户的隐含向量\n",
    "qi =  np.zeros((n_items,K))    \n",
    "pu =  np.zeros((n_users,K))   \n",
    "\n",
    "\n",
    "for uid in range(n_users):  #对每个用户\n",
    "    pu[uid] = np.reshape(random((K,1))/10*(np.sqrt(K)),K)\n",
    "       \n",
    "for iid in range(n_items):  #对每个item\n",
    "    qi[iid] = np.reshape(random((K,1))/10*(np.sqrt(K)),K)\n",
    "\n",
    "#所有用户的平均打分\n",
    "mu = df_triplet['rate'].mean()  #average rating"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 根据当前参数，预测用户uid对Item（i_id）的打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def svd_pred(uid, iid):  \n",
    "    score = mu + bi[iid] + bu[uid] + np.sum(qi[iid]* pu[uid])  \n",
    "        \n",
    "    #将打分范围控制在1-5之间\n",
    "    #if score>5:  \n",
    "        #score = 5  \n",
    "    #elif score<1:  \n",
    "        #score = 1  \n",
    "        \n",
    "    return score  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 0-th  step is running\n",
      "the rmse of this step on train data is  [0.41726412]\n",
      "The 1-th  step is running\n",
      "the rmse of this step on train data is  [0.41693142]\n",
      "The 2-th  step is running\n",
      "the rmse of this step on train data is  [0.41652781]\n",
      "The 3-th  step is running\n",
      "the rmse of this step on train data is  [0.41625356]\n",
      "The 4-th  step is running\n",
      "the rmse of this step on train data is  [0.41602549]\n",
      "The 5-th  step is running\n",
      "the rmse of this step on train data is  [0.41551084]\n",
      "The 6-th  step is running\n",
      "the rmse of this step on train data is  [0.41528424]\n",
      "The 7-th  step is running\n",
      "the rmse of this step on train data is  [0.41493812]\n",
      "The 8-th  step is running\n",
      "the rmse of this step on train data is  [0.41480131]\n",
      "The 9-th  step is running\n",
      "the rmse of this step on train data is  [0.41452565]\n",
      "The 10-th  step is running\n",
      "the rmse of this step on train data is  [0.41417427]\n",
      "The 11-th  step is running\n",
      "the rmse of this step on train data is  [0.41371344]\n",
      "The 12-th  step is running\n",
      "the rmse of this step on train data is  [0.41368632]\n",
      "The 13-th  step is running\n",
      "the rmse of this step on train data is  [0.41315947]\n",
      "The 14-th  step is running\n",
      "the rmse of this step on train data is  [0.41318469]\n",
      "The 15-th  step is running\n",
      "the rmse of this step on train data is  [0.41278352]\n",
      "The 16-th  step is running\n",
      "the rmse of this step on train data is  [0.41270445]\n",
      "The 17-th  step is running\n",
      "the rmse of this step on train data is  [0.41241999]\n",
      "The 18-th  step is running\n",
      "the rmse of this step on train data is  [0.41225973]\n",
      "The 19-th  step is running\n",
      "the rmse of this step on train data is  [0.41212802]\n"
     ]
    }
   ],
   "source": [
    "#gamma：为学习率\n",
    "#Lambda：正则参数\n",
    "#steps：迭代次数\n",
    "\n",
    "steps=20\n",
    "gamma=0.04\n",
    "Lambda=0.15\n",
    "\n",
    "#总的打分记录数目\n",
    "n_records = df_triplet.shape[0]\n",
    "\n",
    "for step in range(steps):  \n",
    "    print ('The ' + str(step) + '-th  step is running' )\n",
    "    rmse_sum=0.0 \n",
    "            \n",
    "    #将训练样本打散顺序\n",
    "    kk = np.random.permutation(n_records)  \n",
    "    for j in range(n_records):  \n",
    "        #每次一个训练样本\n",
    "        line = kk[j]  \n",
    "        \n",
    "        uid = users_index [df_triplet.iloc[line]['user']]\n",
    "        iid = items_index [df_triplet.iloc[line]['song']]\n",
    "    \n",
    "        rating  = df_triplet.iloc[line]['rate']\n",
    "                \n",
    "        #预测残差\n",
    "        eui = rating - svd_pred(uid, iid)  \n",
    "        #残差平方和\n",
    "        rmse_sum += eui**2  \n",
    "                \n",
    "        #随机梯度下降，更新\n",
    "        bu[uid] += gamma * (eui - Lambda * bu[uid])  \n",
    "        bi[iid] += gamma * (eui - Lambda * bi[iid]) \n",
    "                \n",
    "        temp = qi[iid]  \n",
    "        qi[iid] += gamma * (eui* pu[uid]- Lambda*qi[iid] )  \n",
    "        pu[uid] += gamma * (eui* temp - Lambda*pu[uid])  \n",
    "            \n",
    "    #学习率递减\n",
    "    gamma=gamma*0.93  \n",
    "    print (\"the rmse of this step on train data is \",np.sqrt(rmse_sum/n_records))  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A method for saving object data to JSON file\n",
    "def save_json(filepath):\n",
    "    dict_ = {}\n",
    "    dict_['mu'] = mu\n",
    "    dict_['K'] = K\n",
    "    \n",
    "    dict_['bi'] = bi.tolist()\n",
    "    dict_['bu'] = bu.tolist()\n",
    "    \n",
    "    dict_['qi'] = qi.tolist()\n",
    "    dict_['pu'] = pu.tolist()\n",
    "\n",
    "    # Creat json and save to file\n",
    "    json_txt = json.dumps(dict_)\n",
    "    with open(filepath, 'w') as file:\n",
    "        file.write(json_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A method for loading data from JSON file\n",
    "def load_json(filepath):\n",
    "    with open(filepath, 'r') as file:\n",
    "        dict_ = json.load(file)\n",
    "\n",
    "        mu = dict_['mu']\n",
    "        K = dict_['K']\n",
    "\n",
    "        bi = np.asarray(dict_['bi'])\n",
    "        bu = np.asarray(dict_['bu'])\n",
    "    \n",
    "        qi = np.asarray(dict_['qi'])\n",
    "        pu = np.asarray(dict_['pu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json('svd_model.json')\n",
    "load_json('svd_model.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对给定用户，推荐物品/计算打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#user：用户\n",
    "#返回推荐items及其打分（DataFrame）\n",
    "def svd_CF_recommend(user):\n",
    "    cur_user_id = users_index[user]\n",
    "    \n",
    "    #训练集中该用户打过分的item\n",
    "    cur_user_items = user_items[cur_user_id]\n",
    "\n",
    "    #该用户对所有item的打分\n",
    "    user_items_scores = np.zeros(n_items)\n",
    "\n",
    "    #预测打分\n",
    "    for i in range(n_items):  # all items \n",
    "        if i not in cur_user_items: #训练集中没打过分\n",
    "            user_items_scores[i] = svd_pred(cur_user_id, i)  #预测打分\n",
    "    \n",
    "    #推荐\n",
    "    #Sort the indices of user_item_scores based upon their value，Also maintain the corresponding score\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(user_items_scores))), reverse=True)\n",
    "    \n",
    "    #Create a dataframe from the following\n",
    "    columns = ['item_id', 'score']\n",
    "    df = pd.DataFrame(columns=columns)\n",
    "         \n",
    "    #Fill the dataframe with top 20 (n_rec_items) item based recommendations\n",
    "    #sort_index = sort_index[0:n_rec_items]\n",
    "    #Fill the dataframe with all items based recommendations\n",
    "    for i in range(0,len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1] \n",
    "        cur_item = list (items_index.keys()) [list (items_index.values()).index (cur_item_index)]\n",
    "            \n",
    "        if ~np.isnan(sort_index[i][0]) and cur_item_index not in cur_user_items:\n",
    "            df.loc[len(df)]=[cur_item, sort_index[i][0]]\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取测试数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>play_count</th>\n",
       "      <th>rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SOCKSGZ12A58A7CA4B</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SOCVTLJ12A6310F0FD</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SODLLYS12A8C13A96B</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SOEGIYH12A6D4FC0E3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SOFRQTD12A81C233C0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  play_count  \\\n",
       "0  4e11f45d732f4861772b2906f81a7d384552ad12  SOCKSGZ12A58A7CA4B           1   \n",
       "1  4e11f45d732f4861772b2906f81a7d384552ad12  SOCVTLJ12A6310F0FD           1   \n",
       "2  4e11f45d732f4861772b2906f81a7d384552ad12  SODLLYS12A8C13A96B           3   \n",
       "3  4e11f45d732f4861772b2906f81a7d384552ad12  SOEGIYH12A6D4FC0E3           1   \n",
       "4  4e11f45d732f4861772b2906f81a7d384552ad12  SOFRQTD12A81C233C0           2   \n",
       "\n",
       "   rate  \n",
       "0     1  \n",
       "1     1  \n",
       "2     3  \n",
       "3     1  \n",
       "4     2  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取测试数据\n",
    "dpath='./Data/'\n",
    "df_triplet_test= pd.read_csv(dpath +'RS_Song_test.csv')\n",
    "df_triplet_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试，并计算评价指标\n",
    "PR、覆盖度、RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4e11f45d732f4861772b2906f81a7d384552ad12 is a new user.\n",
      "\n",
      "da681f423e8574d2581534fd138eae264dea0f5c is a new user.\n",
      "\n",
      "371e9aff6b9f644e7598dbc4dd5640b1544b97f3 is a new user.\n",
      "\n",
      "c0ff0f1c93f67c1fb372b36b1b08bb4c76bead7d is a new user.\n",
      "\n",
      "13ce57b3a25ef63fa614335fd838e8024c42ec17 is a new user.\n",
      "\n",
      "6a944bfe30ae8d6b873139e8305ae131f1607d5f is a new user.\n",
      "\n",
      "45f06d9d15adbd8575a63e848b1f1c202afbf308 is a new user.\n",
      "\n",
      "9acce6b22c8ae6adbd9e0933fb0a374b149e8e88 is a new user.\n",
      "\n",
      "283882c3d18ff2ad0e17124002ec02b847d06e9a is a new user.\n",
      "\n",
      "ee7aa84c164038c963cfd02a7e52a5598aa470c3 is a new user.\n",
      "\n",
      "b048f21afd5e7467f187bf9f9d413e97c32313a9 is a new user.\n",
      "\n",
      "04043d5716a2359f49f62d13c7c3d3e72b28f520 is a new user.\n",
      "\n",
      "7e40c676afe9ffcb80dfcb62e2acffd26ae21996 is a new user.\n",
      "\n",
      "ee30810179c611d32705fe0b71333dcb8703b30a is a new user.\n",
      "\n",
      "67874d1a189c83326c529e554be6f7acf55effae is a new user.\n",
      "\n",
      "e5b4f068ec1446a74564b4b8b8d13dcea7d27840 is a new user.\n",
      "\n",
      "1aa4fd215aadb160965110ed8a829745cde319eb is a new user.\n",
      "\n",
      "0091e0326c4c034cc04be6454742912845740a1f is a new user.\n",
      "\n",
      "30cc99a1b03a19cc162b286c76047e58371ffb3d is a new user.\n",
      "\n",
      "6ccd111af9b4baa497aacd6d1863cbf5a141acc6 is a new user.\n",
      "\n",
      "91ee487b65406cec50462071a41a83b5d6e99342 is a new user.\n",
      "\n",
      "2231cb435771a1a621ec44e95cdd28b81fad3288 is a new user.\n",
      "\n",
      "abaf265b7dc7d19732cde4e55e04a1ae65514e03 is a new user.\n",
      "\n",
      "4be63160ad51eb4c19c3ba70bf097b729d929650 is a new user.\n",
      "\n",
      "358e0fdc1b9d9a219ccea844b58bd8e6e35f0533 is a new user.\n",
      "\n",
      "6d625c6557df84b60d90426c0116138b617b9449 is a new user.\n",
      "\n",
      "330a7b1168a732742b572ba38a186c5b951b7f2d is a new user.\n",
      "\n",
      "81f0878d4707a39b4aecb412662f901e706adb35 is a new user.\n",
      "\n",
      "083a2a59603a605275107c00812a811526c2a0af is a new user.\n",
      "\n",
      "9c859962257112ad523f1d3c121d35191daa6d2b is a new user.\n",
      "\n",
      "dc02d3c4fb534ca1406f720c70678e5858a5ca0a is a new user.\n",
      "\n",
      "95926d03f275f4a3f04f068c2015e101a93a9292 is a new user.\n",
      "\n",
      "53f5a96ce4fe0b735815c877e225225e95061b1b is a new user.\n",
      "\n",
      "d9e5124935aebeb6b9b169df2d28e1fc658d3566 is a new user.\n",
      "\n",
      "debd74c1999b195eeddf09bc0af751c93a52c205 is a new user.\n",
      "\n",
      "7e89a43dc229de4259b5a36053de255a6247a773 is a new user.\n",
      "\n",
      "43e241eaa4d079ab2bdbd30231695f5563583af4 is a new user.\n",
      "\n",
      "d13609d62db6df876d3cc388225478618bb7b912 is a new user.\n",
      "\n",
      "ed3e56f28c9b1f6dd18270cf3c29f764ae8e4712 is a new user.\n",
      "\n",
      "7e41241058af7545be75fb1e38504520e88a0ceb is a new user.\n",
      "\n",
      "48567d388c6a7dda0e9d0a7b6648bdb42440475c is a new user.\n",
      "\n",
      "82cd7de99ecda208dcd18346892859f35daf0520 is a new user.\n",
      "\n",
      "360dfc77809fc2f4a5863eb59bcf4191de6635ef is a new user.\n",
      "\n",
      "dd29009e6148c8c21544c27e7c074b4601ba54e5 is a new user.\n",
      "\n",
      "b055cc3ae8159fe4bc656dc5631cc6659fc5cf72 is a new user.\n",
      "\n",
      "5ea608df0357ec4fda191cb9316fe8e6e65e3777 is a new user.\n",
      "\n",
      "a15075a926c1998d91940f118342ba8356efc7d4 is a new user.\n",
      "\n",
      "0b19fe0fad7ca85693846f7dad047c449784647e is a new user.\n",
      "\n",
      "70caceccaa745b6f7bc2898a154538eb1ada4d5a is a new user.\n",
      "\n",
      "fe2d77de7e57f3b3eedcf473545110b13ca03426 is a new user.\n",
      "\n",
      "4c6426c9c734c23ec0c35154437fc132b862068c is a new user.\n",
      "\n",
      "1820cfffd52cad7b3af398f379524d51579655d2 is a new user.\n",
      "\n",
      "972cce803aa7beceaa7d0039e4c7c0ff097e4d55 is a new user.\n",
      "\n",
      "d8aace9308e530e0f1ef2a85258b62f8245be423 is a new user.\n",
      "\n",
      "da3890400751de76f0f05ef0e93aa1cd898e7dbc is a new user.\n",
      "\n",
      "0a4c3c6999c74af7d8a44e96b44bf64e513c0f8b is a new user.\n",
      "\n",
      "5e53fe2ce11639e43554127e893dfe3b1b8cb1be is a new user.\n",
      "\n",
      "b85e44249a1d26a378c0757442452525ecfa1abf is a new user.\n",
      "\n",
      "c012ec364329bb08cbe3e62fe76db31f8c5d8ec3 is a new user.\n",
      "\n",
      "8c800d6ece52acb6aff6ab60d991d090c84aabdc is a new user.\n",
      "\n",
      "87024dfca766888f3eb47b2f1b71e2eff60ddb86 is a new user.\n",
      "\n",
      "11089077167c838f036384933d6c0ecafce6814b is a new user.\n",
      "\n",
      "4be305e02f4e72dad1b8ac78e630403543bab994 is a new user.\n",
      "\n",
      "952130c8ae0527458c75471812dbb5c506b46d5d is a new user.\n",
      "\n",
      "315213e4a0e89abdd69633c46f83b6e1dc0e8d87 is a new user.\n",
      "\n",
      "fef771ab021c200187a419f5e55311390f850a50 is a new user.\n",
      "\n",
      "25f83a09541073b1bde62e6e513cb938d3b148de is a new user.\n",
      "\n",
      "94d3dcdd520f1f2f1daec98d1f3736bcff751654 is a new user.\n",
      "\n",
      "54d07c4eff369f457c7a5d1c0b9b6c48feb949f4 is a new user.\n",
      "\n",
      "bb0d8c40d8932c5960e8e4443aa391adb9dfe9da is a new user.\n",
      "\n",
      "71dac99494dd7439a105d14f7597ae706b19c9ed is a new user.\n",
      "\n",
      "96f7b4f800cafef33eae71a6bc44f7139f63cd7a is a new user.\n",
      "\n",
      "2102cfa15b3702123d179c1559717e1886170139 is a new user.\n",
      "\n",
      "6a58f480d522814c087fd3f8c77b3f32bb161f9d is a new user.\n",
      "\n",
      "bb279e21a433803eb4a3dc5d31d2fed80d06fc98 is a new user.\n",
      "\n",
      "fc05f377863a77d7784b02de2cc06cdecb85968b is a new user.\n",
      "\n",
      "e5b1a35349132afe4f96dd6b39cdc4d94031eb1b is a new user.\n",
      "\n",
      "b7948bd0aab3932852dce33f89604a85736a449a is a new user.\n",
      "\n",
      "1fbb0f6de552dec6eff67288fbaf3efeae2a2427 is a new user.\n",
      "\n",
      "dae0bbb6e964a8f5495fa6edfb7321437b28b5a1 is a new user.\n",
      "\n",
      "7b01590cce3cb5c11efb729f55551b9ce252a00d is a new user.\n",
      "\n",
      "65c23b32ecf7636bd62af8f928967cd0f23b00d8 is a new user.\n",
      "\n",
      "c1912062175dc4b3ea5a3a0cdb963c704bb9c881 is a new user.\n",
      "\n",
      "6b2787bcf47cfea75b734729cdaddd700137ad5a is a new user.\n",
      "\n",
      "63aa3e991a33c309016051541b8084ff0aec7284 is a new user.\n",
      "\n",
      "c15570ec9ccc9f4093fc51dba952de0558c74ae7 is a new user.\n",
      "\n",
      "b5186694bb2406962edb3a8042b535449627fa37 is a new user.\n",
      "\n",
      "79a1ba30cec2ee1509f3c2df697916fe901b37e9 is a new user.\n",
      "\n",
      "f6ae5e682750e815c1709ca99138d03b039839d6 is a new user.\n",
      "\n",
      "e0152edeec1154b620b083fbb095f1f0f4de5e2e is a new user.\n",
      "\n",
      "edbeebbe98aeceb8d38b1de33ac1b1e201041284 is a new user.\n",
      "\n",
      "23ec7765590efc05002ccf6551cbadaa953c2069 is a new user.\n",
      "\n",
      "f8544ba8ff908f44d61f5d9d17c213423c1fc782 is a new user.\n",
      "\n",
      "2b04ee89ef77af7d11abf3077e366d1599656232 is a new user.\n",
      "\n",
      "de32503addf63b2ca79e523770bc9c790c3b4d63 is a new user.\n",
      "\n",
      "1d36c586adf56c1be114fb173bf23ececa79e9c7 is a new user.\n",
      "\n",
      "814dda72451239575dc8675ad72e0b612a3bba5e is a new user.\n",
      "\n",
      "e0314b2a3e48610bedb47f4663916170fb6188aa is a new user.\n",
      "\n",
      "f36597bc11ce99d965f4db2e065cf9d648bda39e is a new user.\n",
      "\n",
      "82b152c09a94dd4f7d4566080a00be441e3efbd9 is a new user.\n",
      "\n",
      "5d5a27d2fe3d45e515e79ddb7f70822ef5cd3600 is a new user.\n",
      "\n",
      "cbbe74dbde73f3b0005f9f1422d9f1e62cb54115 is a new user.\n",
      "\n",
      "7c9b8125c230d20b19317aa06a4b3f8415057f88 is a new user.\n",
      "\n",
      "e3fce0e4f316567685119c9af7d04d67c84f0a46 is a new user.\n",
      "\n",
      "05be6a588cc454d1e400a358c562807aeec8c054 is a new user.\n",
      "\n",
      "1198e6c8673f603114e7de4b1045a6cc6e80c897 is a new user.\n",
      "\n",
      "8ad6a386e7b601c0072c337ec6f86d084d1ba556 is a new user.\n",
      "\n",
      "4bcc4cfd9acf7e19bbccd398f8503ba79fb66513 is a new user.\n",
      "\n",
      "a4493d7bb8c07dc19e4208c8ed6925ec40300d36 is a new user.\n",
      "\n",
      "2f72cbcbf49aac438f614626d86c5e93efb82206 is a new user.\n",
      "\n",
      "9af7bb657715f4b8bb63588fc8c8a3ec46cbdb6e is a new user.\n",
      "\n",
      "3a6553c25f5dce3dbbbedc429496d40bf913e354 is a new user.\n",
      "\n",
      "7ffc14a55b6256c9fa73fc5c5761d210deb7f738 is a new user.\n",
      "\n",
      "6086417a706e339f64f47752b474e9c9596ff66d is a new user.\n",
      "\n",
      "2b77760cb7cf72d64dd6d5bdf8a5c195a044a9b3 is a new user.\n",
      "\n",
      "0084ecc8a2b3b0a371b968ded8b92d3d8525fd64 is a new user.\n",
      "\n",
      "d91ca966f809ef4a77999b031c90fcd5253fecc0 is a new user.\n",
      "\n",
      "2bc9df66877af6eddb7103859a574615f0a8aa91 is a new user.\n",
      "\n",
      "8ce501388bc1fa1c67f7563d9b2e13ce4554e375 is a new user.\n",
      "\n",
      "82fed477acd885cc428c29f1423f1929582d0643 is a new user.\n",
      "\n",
      "0c68affd09224b5231f656c31aec29505da2f661 is a new user.\n",
      "\n",
      "f3825861e5575cf4206f883630754fa30c80721e is a new user.\n",
      "\n",
      "e3dc1a577d1432b6406f3bed18d92fd6830ed69e is a new user.\n",
      "\n",
      "3f1037fb3a74f351cd61fff169a54896d2be06f9 is a new user.\n",
      "\n",
      "e3d45fa4071c85b044191cabf0284e74c744a037 is a new user.\n",
      "\n",
      "ffe5ad43c24d81878621185e164043a6e49b2fe4 is a new user.\n",
      "\n",
      "ffe2ec5b72cddb8537ad7f0ac191624f8ae2c8dc is a new user.\n",
      "\n",
      "eaa2b3c9e086a662ab15e10ca6211a3207f40a50 is a new user.\n",
      "\n",
      "69963f38f75cedf4029510ba20351fec7bdaede9 is a new user.\n",
      "\n",
      "d446e7c921a102071026003e6baf3175afc535e8 is a new user.\n",
      "\n",
      "cd58f3d02bfa9e5df95971cde778593012241236 is a new user.\n",
      "\n",
      "1a6916b63039c3c56c62de345c284c11a075eb96 is a new user.\n",
      "\n",
      "b6294fa6a76a84789d038b7133ddc308391bacc6 is a new user.\n",
      "\n",
      "9c6ebf1d5ba38bcb577149b22e19448f655f6252 is a new user.\n",
      "\n",
      "32e46bf89e523d8261aab60b31a766a4d24af83e is a new user.\n",
      "\n",
      "6049f073c9b55e1f0995902414dd64f41357ffa3 is a new user.\n",
      "\n",
      "d2396f84238693721349e3dfb9e109bb67b93fca is a new user.\n",
      "\n",
      "ab8fcf84d35479e7a2c88416040e67a78217fbda is a new user.\n",
      "\n",
      "b3dd0cfd6801e5af3a1a42acd9055f97c0112ad8 is a new user.\n",
      "\n",
      "ae552344f08d59c11b0a0b217f66b17ba9b37ba9 is a new user.\n",
      "\n",
      "821475e83c0f0cadfe0a30c3eb8034ff5c45f3af is a new user.\n",
      "\n",
      "c9ccbdf63666eb4b570bb9103f371346e2c3b378 is a new user.\n",
      "\n",
      "8cb51abc6bf8ea29341cb070fe1e1af5e4c3ffcc is a new user.\n",
      "\n",
      "2fa58dbe45db621eadad025c7bf3e30957eee321 is a new user.\n",
      "\n",
      "ae30f0628af54f394bc94e29c934913d1ef7cff6 is a new user.\n",
      "\n",
      "e8f6a8d06b0096737dec1a9f44c3d48cd9e5e4b8 is a new user.\n",
      "\n",
      "39797b8781fee9c03a218afddda0a4759dabc937 is a new user.\n",
      "\n",
      "2d1d1596c8d162b33ce8bdb575dae8073afa1086 is a new user.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#统计总的用户\n",
    "unique_users_test = df_triplet_test['user'].unique()\n",
    "\n",
    "#为每个用户推荐的item的数目\n",
    "n_rec_items = 10\n",
    "\n",
    "#性能评价参数初始化，用户计算Percison和Recall\n",
    "n_hits = 0\n",
    "n_total_rec_items = 0\n",
    "n_test_items = 0\n",
    "\n",
    "#所有被推荐商品的集合（对不同用户），用于计算覆盖度\n",
    "all_rec_items = set()\n",
    "\n",
    "#残差平方和，用与计算RMSE\n",
    "rss_test = 0.0\n",
    "\n",
    "#对每个测试用户\n",
    "for user in unique_users_test:\n",
    "    #测试集中该用户打过分的电影（用于计算评价指标的真实值）\n",
    "    if user not in users_index:   #user在训练集中没有出现过，新用户不能用协同过滤\n",
    "        print(str(user) + ' is a new user.\\n')\n",
    "        continue\n",
    "   \n",
    "    user_records_test= df_triplet_test[df_triplet_test.user== user]\n",
    "    \n",
    "    #对每个测试用户，计算该用户对训练集中未出现过的商品的打分，并基于该打分进行推荐（top n_rec_items）\n",
    "    #返回结果为DataFrame\n",
    "    rec_items = svd_CF_recommend(user)\n",
    "    for i in range(n_rec_items):\n",
    "        item = rec_items.iloc[i]['item_id']\n",
    "        \n",
    "        if item in user_records_test['song'].values:\n",
    "            n_hits += 1\n",
    "        all_rec_items.add(item)\n",
    "    \n",
    "    #计算rmse\n",
    "    for i in range(user_records_test.shape[0]):\n",
    "        item = user_records_test.iloc[i]['song']\n",
    "        score = user_records_test.iloc[i]['rate']\n",
    "        \n",
    "        df1 = rec_items[rec_items.item_id == item]\n",
    "        if(df1.shape[0] == 0): #item不在推荐列表中，可能是新item在训练集中没有出现过，或者该用户已经打过分新item不能被协同过滤推荐\n",
    "            print(str(item) + ' is a new song or  user ' + str(user) +' already rated it.\\n')\n",
    "            continue\n",
    "        pred_score = df1['score'].values[0]\n",
    "        rss_test += (pred_score - score)**2     #残差平方和\n",
    "    \n",
    "    #推荐的item总数\n",
    "    n_total_rec_items += n_rec_items\n",
    "    \n",
    "    #真实item的总数\n",
    "    n_test_items += user_records_test.shape[0]\n",
    "\n",
    "#Precision & Recall\n",
    "precision = n_hits / (1.0*n_total_rec_items)\n",
    "recall = n_hits / (1.0*n_test_items)\n",
    "\n",
    "#覆盖度：推荐商品占总需要推荐商品的比例\n",
    "coverage = len(all_rec_items) / (1.0* n_items)\n",
    "\n",
    "#打分的均方误差\n",
    "rmse=np.sqrt(rss_test / df_triplet_test.shape[0])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0125"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3268539150447036"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "额。。结果是0还是说保留小数点？不应该啊。。。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
